Overview

This notebook tests our preregistered hypothesis that there is an effect of condition on the probability of correctly recalling a studied item on a test.

Setup

library(BayesFactor)
Loading required package: coda
Loading required package: Matrix
************
Welcome to BayesFactor 0.9.12-4.2. If you have questions, please contact Richard Morey (richarddmorey@gmail.com).

Type BFManual() to open the manual.
************
library(dplyr)

Attaching package: 'dplyr'
The following objects are masked from 'package:stats':

    filter, lag
The following objects are masked from 'package:base':

    intersect, setdiff, setequal, union
library(data.table)

Attaching package: 'data.table'
The following objects are masked from 'package:dplyr':

    between, first, last
library(forcats)
library(ggplot2)
library(purrr)

Attaching package: 'purrr'
The following object is masked from 'package:data.table':

    transpose
library(readr)
library(stringr)
library(tidyr)

Attaching package: 'tidyr'
The following objects are masked from 'package:Matrix':

    expand, pack, unpack
library(extrafont)
Registering fonts with R
library(wesanderson)
library(tikzDevice)
library(fst)
library(brms)
Loading required package: Rcpp
Registered S3 method overwritten by 'xts':
  method     from
  as.zoo.xts zoo 
Loading 'brms' package (version 2.12.0). Useful instructions
can be found by typing help('brms'). A more detailed introduction
to the package is available through vignette('brms_overview').

Attaching package: 'brms'
The following object is masked from 'package:stats':

    ar
library(ggridges)

Attaching package: 'ggridges'
The following object is masked from 'package:ggplot2':

    scale_discrete_manual
library(lme4)

Attaching package: 'lme4'
The following object is masked from 'package:brms':

    ngrps
library(future)

plan(multiprocess)

# font_import() # Run once to populate the R font database with the fonts on the system
loadfonts(quiet = TRUE)

theme_poster <- theme_light(base_size = 14) +
            theme(text = element_text(family = 'Merriweather Sans'),
                  strip.text = element_text(colour = "black")) 

theme_paper <- theme_classic(base_size = 12) + 
  theme(axis.text = element_text(colour = "black"),
        panel.grid.major.y = element_line(colour = "grey92"))

condition_colours <- wes_palette("Darjeeling1", n = 5)
condition_colours <- condition_colours[c(1,3)]

knitr::opts_chunk$set(fig.width=16, fig.height=16)

set.seed(0)

Prepare data

s2_test1 <- read_fst(file.path("..", "data", "processed", "s2", "s2_test1.fst"))
s2_test2 <- read_fst(file.path("..", "data", "processed", "s2", "s2_test2.fst"))
test_full <- bind_rows(s2_test1, s2_test2) %>%
  mutate(block = as.factor(block),
         condition = as.factor(str_to_title(condition)))
Warning in bind_rows_(x, .id): Unequal factor levels: coercing to character
Warning in bind_rows_(x, .id): binding character and factor vector,
coercing into character vector

Warning in bind_rows_(x, .id): binding character and factor vector,
coercing into character vector
test_studied_full <- filter(test_full, studied)

Filter out excluded participant:

s2_exclude <- read_fst(file.path("..", "data", "processed", "s2", "s2_exclude.fst"))

test <- anti_join(test_full, s2_exclude, by = "subject") %>% droplevels()
Warning: Column `subject` joining factor and character vector, coercing
into character vector
test_studied <- anti_join(test_studied_full, s2_exclude, by = "subject") %>% droplevels()
Warning: Column `subject` joining factor and character vector, coercing
into character vector

Test accuracy

According to our hypothesis, test accuracy should be higher in the fact blocks than in the default blocks, since facts are repeated with a more appropriate schedule from the start.

The plot below shows the percentage of correct answers on the test in each block, split by condition order. Only test items that appeared during the learning session are included.

test_acc_studied <- test_studied %>%
  group_by(subject, block, condition) %>%
  summarise(p_corr = mean(correct)) %>%
    mutate(condition_order = case_when(
    block == 1 && condition == "Default" ~ "Default-Fact",
    block == 2 && condition == "Fact" ~ "Default-Fact",
    block == 1 && condition == "Fact" ~ "Fact-Default",
    block == 2 && condition == "Default" ~ "Fact-Default"
  ))
ggplot(test_acc_studied, aes(x = block, y = p_corr)) +
  facet_grid(~ condition_order) +
  geom_violin() +
  geom_boxplot(width = 0.2) +
  labs(x = "Block", y = "Accuracy") +
  theme_poster

Make the plot shown in the paper:

p <- test_acc_studied %>%
  ungroup() %>%
  mutate(condition = stringr::str_to_title(condition)) %>%
  ggplot(aes(y = p_corr, x = condition, group = condition)) +
  geom_jitter(width = 0.1, height = 0, alpha = 0.3, aes(colour = condition)) +
  geom_violin(fill = NA, width = 0.75) +
  geom_boxplot(width = 0.2, outlier.shape = NA, fill = NA) +
  guides(colour = FALSE) +
  expand_limits(y = 0) +
  scale_y_continuous(labels = scales::percent_format()) +
  scale_colour_manual(values = condition_colours) +
  labs(x = NULL,
       y = "Test accuracy") +
  theme_paper

saveRDS(p, "../output/test-accuracy.rds")

p

Make plot for presentation:

theme_presentation <- theme_light(base_size = 18) +
  theme(text = element_text(family = "Merriweather Sans", colour = "black"),
        strip.text = element_text(colour = "black"),
        plot.title = element_text(size = 14, hjust = 0.5),
        axis.title = element_text(size = 14),
        rect = element_rect(fill = "transparent"),
        plot.background = element_rect(fill = "transparent", color = NA),
        panel.background  = element_rect(fill = "white", colour = "grey70")
  ) 

test_acc_studied %>%
  ggplot(aes(y = p_corr, x = condition, group = condition)) +
  geom_jitter(width = 0.1, height = 0, alpha = 0.3, aes(colour = condition)) +
  geom_violin(fill = NA, width = 0.75) +
  geom_boxplot(width = 0.2, outlier.shape = NA, fill = NA) +
  guides(colour = FALSE) +
  expand_limits(y = 0) +
  scale_x_discrete(labels = c("Cold start", "Warm start")) +
  scale_y_continuous(labels = scales::percent_format()) +
  scale_colour_manual(values = condition_colours) +
  labs(x = NULL,
       y = "Test accuracy") +
  theme_presentation

ggsave("../output/test-accuracy-presentation.png", device = "png", width = 4, height = 3, bg = "transparent")

Logistic mixed-effects model

As preregistered, we will evaluate the strength of evidence for an effect of condition on the probability of answering test items correctly using a logistic mixed effects model, with main effects for condition and block, and random intercepts for items and subjects. The fixed effects will be sum-to-zero contrast-coded and will have Cauchy(0,1) priors.

We will then perform a Savage-Dickey density ratio test to establish the strength of evidence for an effect of condition.

Model fitting

Set up contrast coding so that the model intercept reflects the grand mean p(correct), allowing us to assess the effects of condition and block separately:

contrasts(test_studied$condition) <- c(-0.5, 0.5)
contrasts(test_studied$condition)
        [,1]
Default -0.5
Fact     0.5
contrasts(test_studied$block) <- c(-0.5, 0.5)
contrasts(test_studied$block)
  [,1]
1 -0.5
2  0.5

Set the priors for the fixed effects:

prior_m1 <- set_prior("cauchy(0, 1)", class = "b")

Fit the model:

m1 <- brm(
  correct ~ condition + block + (1 | subject) + (1 | fact_id),
  family = bernoulli,
  data = test_studied,
  prior = prior_m1,
  chains = 4,
  iter = 10000,
  save_all_pars = TRUE,
  sample_prior = TRUE,
  future = TRUE,
  seed = 0,
  file = "model_fits/m1"
)

Model inspection

Ensure that MCMC went as expected. The caterpillar plots look normal, and there is no evidence of autocorrelation in the chains.

mcmc_plot(m1, pars = c("condition", "block"), type = "trace")
No divergences to plot.

mcmc_plot(m1, pars = c("condition", "block"), type = "acf_bar")

Model interpretation

Model summary:

summary(m1)
 Family: bernoulli 
  Links: mu = logit 
Formula: correct ~ condition + block + (1 | subject) + (1 | fact_id) 
   Data: test_studied (Number of observations: 2274) 
Samples: 4 chains, each with iter = 10000; warmup = 5000; thin = 1;
         total post-warmup samples = 20000

Group-Level Effects: 
~fact_id (Number of levels: 60) 
              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept)     0.48      0.09     0.32     0.67 1.00     8642    12770

~subject (Number of levels: 68) 
              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept)     0.62      0.09     0.46     0.82 1.00     8443    12525

Population-Level Effects: 
           Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
Intercept      1.36      0.12     1.13     1.59 1.00    14429    14117
condition1     0.42      0.11     0.21     0.63 1.00    43086    13835
block1         0.39      0.11     0.18     0.59 1.00    45202    14866

Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
and Tail_ESS are effective sample size measures, and Rhat is the potential
scale reduction factor on split chains (at convergence, Rhat = 1).

Visualise the model’s population-level estimates, along with their 95% credible intervals:

mcmc_plot(m1, pars = c("condition", "block"), type = "areas", prob = .95)
Warning: `expand_scale()` is deprecated; use `expansion()` instead.

The model makes the following predictions, based on its fixed effects.

# Get fitted values
new_data <- crossing(condition = levels(test_studied$condition), block = levels(test_studied$block))
fit_m1 <- data.table(fitted(m1,
                            newdata = new_data,
                            re_formula = NA,
                            summary = FALSE))
setnames(fit_m1, as.character(interaction(new_data$condition, new_data$block)))

fit_m1_summary <- data.table(fitted(m1,
                                    newdata = new_data,
                                    re_formula = NA))
setnames(fit_m1_summary, c("fit", "se", "lower", "upper"))
fit_m1_summary <- cbind(new_data, fit_m1_summary)

Overall response accuracy (across blocks and conditions):

quantile((fit_m1$Default.1 + fit_m1$Default.2 + fit_m1$Fact.1 + fit_m1$Fact.2)/4,
         probs = c(.5, .025, .975))
      50%      2.5%     97.5% 
0.7906751 0.7524486 0.8266936

Difference in response accuracy between Fact condition and Default condition (across blocks):

fact_vs_default <- (fit_m1$Fact.1 + fit_m1$Fact.2)/2 - (fit_m1$Default.1 + fit_m1$Default.2)/2
quantile(fact_vs_default, probs = c(.5, .025, .975))
       50%       2.5%      97.5% 
0.06848663 0.03424775 0.10466724

Difference in response accuracy between block 2 and block 1 (across conditions):

block2_vs_block1 <- (fit_m1$Default.2 + fit_m1$Fact.2)/2 - (fit_m1$Default.1 + fit_m1$Fact.1)/2
quantile(block2_vs_block1, probs = c(.5, .025, .975))
       50%       2.5%      97.5% 
0.06257268 0.02829405 0.09857257

Hypothesis testing

Now we can test the hypothesis that there is an effect of condition against the null hypothesis that there is not. We do this by computing the density ratio between the posterior and the prior of the condition coefficient at 0. The prior is a Cauchy(0,1) distribution, which means we can calculate its density at 0 analytically. The posterior density is estimated on the basis of the posterior samples from the model.

prior_density_at_zero <- dcauchy(0, 0, 1)
m1_posterior_samples <- posterior_samples(m1, pars = "b_condition1")$b_condition1
posterior_density_at_zero <- density_ratio(m1_posterior_samples, point = 0)


xval <- seq(-2, 2, by = 0.01)
plot_prior <- dcauchy(xval, location = 0, scale = 1)
plot_posterior <- density_ratio(x = m1_posterior_samples, y = NULL, point = xval) # Returns the density of the posterior at each point in x
m1_density <- tibble(xval = xval, Prior = plot_prior, Posterior = plot_posterior)
gather(m1_density, -xval, key = "Type", value = "density") %>%
  mutate(density = ifelse(density < 0, 0, density)) %>%
  ggplot(aes(x = xval, y = density, fill = Type)) +
  geom_area(position = "identity", colour = "black", alpha = 0.75) +
  geom_vline(xintercept = 0, lty = 3) +
  annotate("point", x = c(0, 0), y = c(prior_density_at_zero, posterior_density_at_zero)) +
  labs(x = "Estimate for coefficient 'condition'", y = NULL, title = "") +
  scale_fill_manual(values = c("#03396c", "#b3cde0"))

ggsave(file.path("..", "output", "sd_ratio.pdf"), device = "pdf", width = 4, height = 3)

The evidence in favour of a non-zero coefficient for condition is evaluated by means of the Savage-Dickey density ratio: the ratio between the prior and posterior density at \(\beta_{condition} = 0\) (the two points on the dotted line in the plot above).

BF_condition <- prior_density_at_zero / posterior_density_at_zero

BF_condition
[1] 181.9398

The evidence in favour of an effect of condition is \(BF_{10}\) = 181.9398.

Sanity checks

Test effect of prior

What if we choose a weaker Cauchy(0, 10) prior?

prior_m1_weak <- set_prior("cauchy(0, 10)", class = "b")

Fit the model:

m1_weak <- brm(
  correct ~ condition + block + (1 | subject) + (1 | fact_id),
  family = bernoulli,
  data = test_studied,
  prior = prior_m1_weak,
  chains = 4,
  iter = 10000,
  save_all_pars = TRUE,
  sample_prior = TRUE,
  future = TRUE,
  seed = 0,
  file = "model_fits/m1_weak"
)
prior_density_at_zero_weak <- dcauchy(0, 0, 10)
m1_weak_posterior_samples <- posterior_samples(m1_weak, pars = "b_condition1")$b_condition1
posterior_density_at_zero_weak <- density_ratio(m1_weak_posterior_samples, point = 0)


xval <- seq(-2, 2, by = 0.01)
plot_prior_weak <- dcauchy(xval, location = 0, scale = 10)
plot_posterior_weak <- density_ratio(x = m1_weak_posterior_samples, y = NULL, point = xval) # Returns the density of the posterior at each point in x
m1_weak_density <- tibble(xval = xval, Prior = plot_prior_weak, Posterior = plot_posterior_weak)
gather(m1_weak_density, -xval, key = "Type", value = "density") %>%
  mutate(density = ifelse(density < 0, 0, density)) %>%
  ggplot(aes(x = xval, y = density, fill = Type)) +
  geom_area(position = "identity", colour = "black", alpha = 0.75) +
  geom_vline(xintercept = 0, lty = 3) +
  annotate("point", x = c(0, 0), y = c(prior_density_at_zero_weak, posterior_density_at_zero_weak)) +
  labs(x = "Estimate for coefficient 'condition'", y = NULL, title = "") +
  scale_fill_manual(values = c("#03396c", "#b3cde0"))

Even with the much weaker prior, the evidence in favour of an effect of condition is still very strong.

BF_condition_weak <- prior_density_at_zero_weak / posterior_density_at_zero_weak

BF_condition_weak
[1] 58.95594

How about a stronger Cauchy(0, 0.5) prior?

prior_m1_strong <- set_prior("cauchy(0, 0.5)", class = "b")

Fit the model:

m1_strong <- brm(
  correct ~ condition + block + (1 | subject) + (1 | fact_id),
  family = bernoulli,
  data = test_studied,
  prior = prior_m1_strong,
  chains = 4,
  iter = 10000,
  save_all_pars = TRUE,
  sample_prior = TRUE,
  future = TRUE,
  seed = 0,
  file = "model_fits/m1_strong"
)
prior_density_at_zero_strong <- dcauchy(0, 0, 0.5)
m1_strong_posterior_samples <- posterior_samples(m1_strong, pars = "b_condition1")$b_condition1
posterior_density_at_zero_strong <- density_ratio(m1_strong_posterior_samples, point = 0)


xval <- seq(-2, 2, by = 0.01)
plot_prior_strong <- dcauchy(xval, location = 0, scale = 0.5)
plot_posterior_strong <- density_ratio(x = m1_strong_posterior_samples, y = NULL, point = xval) # Returns the density of the posterior at each point in x
m1_strong_density <- tibble(xval = xval, Prior = plot_prior_strong, Posterior = plot_posterior_strong)
gather(m1_strong_density, -xval, key = "Type", value = "density") %>%
  mutate(density = ifelse(density < 0, 0, density)) %>%
  ggplot(aes(x = xval, y = density, fill = Type)) +
  geom_area(position = "identity", colour = "black", alpha = 0.75) +
  geom_vline(xintercept = 0, lty = 3) +
  annotate("point", x = c(0, 0), y = c(prior_density_at_zero_strong, posterior_density_at_zero_strong)) +
  labs(x = "Estimate for coefficient 'condition'", y = NULL, title = "") +
  scale_fill_manual(values = c("#03396c", "#b3cde0"))

With the stronger prior, the evidence in favour of an effect of condition is again very strong.

BF_condition_strong <- prior_density_at_zero_strong / posterior_density_at_zero_strong

BF_condition_strong
[1] 284.4968

Test effect of outliers

The analysis above did not include data from excluded participants (see Prepare data section). Verify that this does not affect the conclusion by also fitting the model to the full dataset.

contrasts(test_studied_full$condition) <- c(-0.5, 0.5)
contrasts(test_studied_full$condition)
        [,1]
Default -0.5
Fact     0.5
contrasts(test_studied_full$block) <- c(-0.5, 0.5)
contrasts(test_studied_full$block)
  [,1]
1 -0.5
2  0.5
m1_full <- brm(
  correct ~ condition + block + (1 | subject) + (1 | fact_id),
  family = bernoulli,
  data = test_studied_full,
  prior = prior_m1,
  chains = 4,
  iter = 10000,
  save_all_pars = TRUE,
  sample_prior = TRUE,
  future = TRUE,
  seed = 0,
  file = "model_fits/m1_full"
)

Check the chains:

mcmc_plot(m1_full, pars = c("condition", "block"), type = "trace")
No divergences to plot.

mcmc_plot(m1_full, pars = c("condition", "block"), type = "acf_bar")

The model estimates look very similar.

summary(m1_full)
 Family: bernoulli 
  Links: mu = logit 
Formula: correct ~ condition + block + (1 | subject) + (1 | fact_id) 
   Data: test_studied_full (Number of observations: 2299) 
Samples: 4 chains, each with iter = 10000; warmup = 5000; thin = 1;
         total post-warmup samples = 20000

Group-Level Effects: 
~fact_id (Number of levels: 60) 
              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept)     0.47      0.09     0.31     0.66 1.00     7966    12855

~subject (Number of levels: 69) 
              Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
sd(Intercept)     0.66      0.09     0.49     0.85 1.00     8900    14053

Population-Level Effects: 
           Estimate Est.Error l-95% CI u-95% CI Rhat Bulk_ESS Tail_ESS
Intercept      1.33      0.12     1.11     1.57 1.00    12585    14927
condition1     0.44      0.11     0.23     0.65 1.00    42349    14539
block1         0.37      0.11     0.16     0.58 1.00    43303    14300

Samples were drawn using sampling(NUTS). For each parameter, Bulk_ESS
and Tail_ESS are effective sample size measures, and Rhat is the potential
scale reduction factor on split chains (at convergence, Rhat = 1).
m1_full_posterior_samples <- posterior_samples(m1_full, pars = "b_condition1")$b_condition1
posterior_density_at_zero_full <- density_ratio(m1_full_posterior_samples, point = 0)

plot_posterior_full <- density_ratio(x = m1_full_posterior_samples, y = NULL, point = xval) # Returns the density of the posterior at each point in x
m1_full_density <- tibble(xval = xval, Prior = plot_prior, Posterior = plot_posterior_full)
gather(m1_full_density, -xval, key = "Type", value = "density") %>%
  mutate(density = ifelse(density < 0, 0, density)) %>%
  ggplot(aes(x = xval, y = density, fill = Type)) +
  geom_area(position = "identity", colour = "black", alpha = 0.75) +
  geom_vline(xintercept = 0, lty = 3) +
  annotate("point", x = c(0, 0), y = c(prior_density_at_zero, posterior_density_at_zero)) +
  labs(x = "Estimate for coefficient 'condition'", y = NULL, title = "") +
  scale_fill_manual(values = c("#03396c", "#b3cde0"))

The Savage-Dickey density ratio is very similar.

BF_condition_full <- prior_density_at_zero / posterior_density_at_zero_full

BF_condition_full
[1] 2068469

Compare to frequentist model

Fit a frequentist glmer with the same model structure. The model summary confirms that we get the same coefficient estimates as with the Bayesian model.

m1_freq <- glmer(correct ~ condition + block + (1 | subject) + (1 | fact_id),
                 family = binomial,
                 data = test_studied)

summary(m1_freq)
Generalized linear mixed model fit by maximum likelihood (Laplace
  Approximation) [glmerMod]
 Family: binomial  ( logit )
Formula: correct ~ condition + block + (1 | subject) + (1 | fact_id)
   Data: test_studied

     AIC      BIC   logLik deviance df.resid 
  2336.0   2364.6  -1163.0   2326.0     2269 

Scaled residuals: 
    Min      1Q  Median      3Q     Max 
-3.8038  0.2721  0.4266  0.5437  1.4019 

Random effects:
 Groups  Name        Variance Std.Dev.
 subject (Intercept) 0.3542   0.5951  
 fact_id (Intercept) 0.2084   0.4565  
Number of obs: 2274, groups:  subject, 68; fact_id, 60

Fixed effects:
            Estimate Std. Error z value Pr(>|z|)    
(Intercept)   1.3474     0.1094  12.321  < 2e-16 ***
condition1    0.4273     0.1070   3.992 6.54e-05 ***
block1        0.3914     0.1065   3.675 0.000238 ***
---
Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1

Correlation of Fixed Effects:
           (Intr) cndtn1
condition1 0.070        
block1     0.016  0.014

Compare SD density ratio to bridge sampling

An alternative method to test for an effect of condition is to compare the marginal likelihood of a model with the predictor to that of a model without it. We can obtain the marginal likelihoods using bridge sampling. This comparison yields a Bayes factor that can be interpreted in the same way.

Fit a second model without condition as a predictor:

m2 <- brm(
  correct ~ block + (1 | subject) + (1 | fact_id),
  family = bernoulli,
  data = test_studied,
  prior = prior_m1,
  chains = 4,
  iter = 10000,
  save_all_pars = TRUE,
  sample_prior = TRUE,
  future = TRUE,
  seed = 0,
  file = "model_fits/m2"
)

Calculate the Bayes factor:

bayes_factor(m1, m2)
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 1
Iteration: 2
Iteration: 3
Iteration: 4
Iteration: 5
Iteration: 6
Estimated Bayes factor in favor of m1 over m2: 212.21710

Again, it’s basically the same as before.

Accumulation of evidence

How does \(BF_{10}\) develop as more participants are added? The following code fits the model repeatedly, each time adding several more participants, and calculates the \(BF_{10}\) for each fit.

subjects <- unique(test_studied$subject)
set_sizes <- seq(1, length(subjects), by = 1)

evidence <- tibble(n = 0, bf = NA, posterior = list(rcauchy(n = 20000, location = 0, scale = 1)))

for (i in set_sizes) {
  
  # Subset the data
  test_studied_subset <- filter(test_studied, subject %in% subjects[1:i])
  
  # Fit the model
  m1_seq <- brm(correct ~ condition + block + (1 | subject) + (1 | fact_id), 
           family = bernoulli,
           data = test_studied_subset,
           prior = prior_m1,
           chains = 4,
           iter = 10000,
           save_all_pars = TRUE,
           sample_prior = TRUE,
           future = TRUE,
           refresh = 0,
           open_progress = FALSE,
           seed = 0,
           file = paste0("model_fits/m1_seq", i))
  
  m1_seq_posterior_samples <- posterior_samples(m1_seq, pars = "b_condition1")$b_condition1
  posterior_density_at_zero_seq <- density_ratio(m1_seq_posterior_samples, point = 0)

  BF_condition_seq <- prior_density_at_zero / posterior_density_at_zero_seq
  
  evidence_seq <- tibble(n = i, bf = BF_condition_seq, posterior = list(m1_seq_posterior_samples))
  evidence <- bind_rows(evidence, evidence_seq)
}

The plot below shows the sequential Bayes factors resulting from this analysis. (Bayes factors calculated from very small samples sizes should obviously be treated with caution.) The accumulation of evidence in favour of an effect of condition is clear. As the BF becomes larger, there are some large upward spikes; this is because by this point there is essentially no posterior mass left at zero, which makes the SD density ratio very sensitive to small fluctuations in the estimated posterior due to the randomness of MCMC sampling.

The days of data collection are marked in the plot. The stopping rule, which was evaluated at the end of each day, required there to be data from at least 40 participants and a BF of 10 or greater in either direction (this boundary is indicated by horizontal lines). At the end of day 2 we had collected data from more than 40 participants, but the BF was still within the boundaries, so we collected data for another day.

seq_plot <- ggplot(filter(evidence, n > 0), aes(n, bf)) +
  annotate("rect", xmin = 0, xmax = 27, ymin = 1/100, ymax = 1000000, fill = "#1b9e77", alpha = 0.2) +
  annotate("rect", xmin = 27, xmax = 42, ymin = 1/100, ymax = 1000000, fill = "#d95f02", alpha = 0.2) +
  annotate("rect", xmin = 42, xmax = 68, ymin = 1/100, ymax = 1000000, fill = "#7570b3", alpha = 0.2) +
  annotate("text", label = c("Day 1", "Day 2", "Day 3"), x = c(13.5, 34.5, 55), y = 0.02) +
  geom_hline(yintercept = c(0.1, 1, 10), lty = c(1, 3, 1)) +
  geom_vline(xintercept = c(0, 27, 42, 68), lty = 2) +
  geom_line() +
  geom_point() +
  scale_y_log10(limits = c(1/100, 1000000), breaks = c(1/100, 1/10, 1, 10, 100, 1000, 10000, 100000, 1000000), labels = function(n) {format(n, scientific = FALSE, drop0trailing = TRUE)}) +
  labs(x = "Sample size", y = "BF10", title = "Sequential Bayes factors in favour of an effect of condition")

seq_plot

The plot below shows how the posterior for the condition coefficient develops as the sample size increases. The dashed vertical line represents the point at which the posterior density is compared to the prior density (shown at the bottom in grey).

unnest_legacy(evidence) %>%
  ggplot(aes(x = posterior, y = n, group = n, fill = n == 0)) +
  geom_vline(xintercept = 0, lty = 2) +
  geom_density_ridges2(scale = 10, size = 0.25) +
  scale_x_continuous(limits = c(-2, 2)) +
  labs(x = "Estimate for coefficient 'condition'", y = "Sample size", title = "Sequential posteriors for 'condition'") +
  scale_fill_manual(values = c("#b3cde0", "#636363")) +
  guides(fill = FALSE)
Picking joint bandwidth of 0.027
Warning: Removed 16332 rows containing non-finite values
(stat_density_ridges).

Session info

sessionInfo()
R version 3.6.3 (2020-02-29)
Platform: x86_64-pc-linux-gnu (64-bit)
Running under: Ubuntu 18.04.5 LTS

Matrix products: default
BLAS:   /usr/lib/x86_64-linux-gnu/blas/libblas.so.3.7.1
LAPACK: /usr/lib/x86_64-linux-gnu/lapack/liblapack.so.3.7.1

locale:
 [1] LC_CTYPE=en_US.UTF-8       LC_NUMERIC=C              
 [3] LC_TIME=nl_NL.UTF-8        LC_COLLATE=en_US.UTF-8    
 [5] LC_MONETARY=nl_NL.UTF-8    LC_MESSAGES=en_US.UTF-8   
 [7] LC_PAPER=nl_NL.UTF-8       LC_NAME=C                 
 [9] LC_ADDRESS=C               LC_TELEPHONE=C            
[11] LC_MEASUREMENT=nl_NL.UTF-8 LC_IDENTIFICATION=C       

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] future_1.13.0          lme4_1.1-21            ggridges_0.5.1        
 [4] brms_2.12.0            Rcpp_1.0.2             fst_0.9.0             
 [7] tikzDevice_0.12        wesanderson_0.3.6      extrafont_0.17        
[10] tidyr_1.0.0            stringr_1.4.0          readr_1.3.1           
[13] purrr_0.3.2            ggplot2_3.3.2          forcats_0.4.0         
[16] data.table_1.12.2      dplyr_0.8.3            BayesFactor_0.9.12-4.2
[19] Matrix_1.2-18          coda_0.19-2           

loaded via a namespace (and not attached):
 [1] minqa_1.2.4          colorspace_1.4-1     ellipsis_0.3.0      
 [4] rsconnect_0.8.15     markdown_1.0         base64enc_0.1-3     
 [7] listenv_0.7.0        rstan_2.19.2         MatrixModels_0.4-1  
[10] DT_0.7               mvtnorm_1.1-1        bridgesampling_0.7-2
[13] codetools_0.2-16     splines_3.6.3        knitr_1.23          
[16] shinythemes_1.1.2    bayesplot_1.7.0      jsonlite_1.6        
[19] nloptr_1.2.1         Rttf2pt1_1.3.7       shiny_1.3.2         
[22] compiler_3.6.3       backports_1.1.4      assertthat_0.2.1    
[25] cli_1.1.0            later_0.8.0          htmltools_0.3.6     
[28] prettyunits_1.0.2    tools_3.6.3          igraph_1.2.4.1      
[31] gtable_0.3.0         glue_1.3.1           reshape2_1.4.3      
[34] vctrs_0.2.2          filehash_2.4-2       nlme_3.1-149        
[37] extrafontdb_1.0      crosstalk_1.0.0      xfun_0.7            
[40] globals_0.12.4       ps_1.3.0             mime_0.7            
[43] miniUI_0.1.1.1       lifecycle_0.1.0      gtools_3.8.1        
[46] MASS_7.3-51.4        zoo_1.8-6            scales_1.0.0        
[49] colourpicker_1.0     hms_0.4.2            promises_1.0.1      
[52] Brobdingnag_1.2-6    parallel_3.6.3       inline_0.3.15       
[55] shinystan_2.5.0      mvnfast_0.2.5        yaml_2.2.0          
[58] pbapply_1.4-0        gridExtra_2.3        loo_2.3.1           
[61] StanHeaders_2.19.0   stringi_1.4.3        dygraphs_1.1.1.6    
[64] boot_1.3-25          pkgbuild_1.0.3       rlang_0.4.4         
[67] pkgconfig_2.0.2      matrixStats_0.55.0   evaluate_0.14       
[70] lattice_0.20-41      labeling_0.3         rstantools_1.5.1    
[73] htmlwidgets_1.3      tidyselect_0.2.5     processx_3.3.1      
[76] plyr_1.8.4           magrittr_1.5         R6_2.4.0            
[79] pillar_1.4.2         withr_2.1.2          xts_0.11-2          
[82] abind_1.4-5          tibble_2.1.3         crayon_1.3.4        
[85] rmarkdown_1.13       grid_3.6.3           callr_3.2.0         
[88] threejs_0.3.1        digest_0.6.19        xtable_1.8-4        
[91] httpuv_1.5.1         stats4_3.6.3         munsell_0.5.0       
[94] shinyjs_1.0
LS0tCnRpdGxlOiAnUHJlZGljdGluZyBBbHBoYTogVGVzdCBvZiBIeXBvdGhlc2lzIDEnCmF1dGhvcjogIk1hYXJ0ZW4gdmFuIGRlciBWZWxkZSIKZGF0ZTogIkxhc3QgdXBkYXRlZDogYHIgU3lzLkRhdGUoKWAiCm91dHB1dDoKICBodG1sX25vdGVib29rOgogICAgc21hcnQ6IG5vCiAgICB0b2M6IHllcwogICAgdG9jX2Zsb2F0OiB5ZXMKICBodG1sX2RvY3VtZW50OgogICAgZGZfcHJpbnQ6IHBhZ2VkCiAgICB0b2M6IHllcwogICAgdG9jX2Zsb2F0OiB5ZXMKZWRpdG9yX29wdGlvbnM6IAogIGNodW5rX291dHB1dF90eXBlOiBpbmxpbmUKLS0tCgoKIyBPdmVydmlldwoKVGhpcyBub3RlYm9vayB0ZXN0cyBvdXIgcHJlcmVnaXN0ZXJlZCBoeXBvdGhlc2lzIHRoYXQgdGhlcmUgaXMgYW4gZWZmZWN0IG9mIGNvbmRpdGlvbiBvbiB0aGUgcHJvYmFiaWxpdHkgb2YgY29ycmVjdGx5IHJlY2FsbGluZyBhIHN0dWRpZWQgaXRlbSBvbiBhIHRlc3QuCgojIFNldHVwCgpgYGB7cn0KbGlicmFyeShCYXllc0ZhY3RvcikKbGlicmFyeShkcGx5cikKbGlicmFyeShkYXRhLnRhYmxlKQpsaWJyYXJ5KGZvcmNhdHMpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShwdXJycikKbGlicmFyeShyZWFkcikKbGlicmFyeShzdHJpbmdyKQpsaWJyYXJ5KHRpZHlyKQpsaWJyYXJ5KGV4dHJhZm9udCkKbGlicmFyeSh3ZXNhbmRlcnNvbikKbGlicmFyeSh0aWt6RGV2aWNlKQpsaWJyYXJ5KGZzdCkKbGlicmFyeShicm1zKQpsaWJyYXJ5KGdncmlkZ2VzKQpsaWJyYXJ5KGxtZTQpCmxpYnJhcnkoZnV0dXJlKQoKcGxhbihtdWx0aXByb2Nlc3MpCgojIGZvbnRfaW1wb3J0KCkgIyBSdW4gb25jZSB0byBwb3B1bGF0ZSB0aGUgUiBmb250IGRhdGFiYXNlIHdpdGggdGhlIGZvbnRzIG9uIHRoZSBzeXN0ZW0KbG9hZGZvbnRzKHF1aWV0ID0gVFJVRSkKCnRoZW1lX3Bvc3RlciA8LSB0aGVtZV9saWdodChiYXNlX3NpemUgPSAxNCkgKwogICAgICAgICAgICB0aGVtZSh0ZXh0ID0gZWxlbWVudF90ZXh0KGZhbWlseSA9ICdNZXJyaXdlYXRoZXIgU2FucycpLAogICAgICAgICAgICAgICAgICBzdHJpcC50ZXh0ID0gZWxlbWVudF90ZXh0KGNvbG91ciA9ICJibGFjayIpKSAKCnRoZW1lX3BhcGVyIDwtIHRoZW1lX2NsYXNzaWMoYmFzZV9zaXplID0gMTIpICsgCiAgdGhlbWUoYXhpcy50ZXh0ID0gZWxlbWVudF90ZXh0KGNvbG91ciA9ICJibGFjayIpLAogICAgICAgIHBhbmVsLmdyaWQubWFqb3IueSA9IGVsZW1lbnRfbGluZShjb2xvdXIgPSAiZ3JleTkyIikpCgpjb25kaXRpb25fY29sb3VycyA8LSB3ZXNfcGFsZXR0ZSgiRGFyamVlbGluZzEiLCBuID0gNSkKY29uZGl0aW9uX2NvbG91cnMgPC0gY29uZGl0aW9uX2NvbG91cnNbYygxLDMpXQoKa25pdHI6Om9wdHNfY2h1bmskc2V0KGZpZy53aWR0aD0xNiwgZmlnLmhlaWdodD0xNikKCnNldC5zZWVkKDApCmBgYAoKIyBQcmVwYXJlIGRhdGEKCmBgYHtyfQpzMl90ZXN0MSA8LSByZWFkX2ZzdChmaWxlLnBhdGgoIi4uIiwgImRhdGEiLCAicHJvY2Vzc2VkIiwgInMyIiwgInMyX3Rlc3QxLmZzdCIpKQpzMl90ZXN0MiA8LSByZWFkX2ZzdChmaWxlLnBhdGgoIi4uIiwgImRhdGEiLCAicHJvY2Vzc2VkIiwgInMyIiwgInMyX3Rlc3QyLmZzdCIpKQp0ZXN0X2Z1bGwgPC0gYmluZF9yb3dzKHMyX3Rlc3QxLCBzMl90ZXN0MikgJT4lCiAgbXV0YXRlKGJsb2NrID0gYXMuZmFjdG9yKGJsb2NrKSwKICAgICAgICAgY29uZGl0aW9uID0gYXMuZmFjdG9yKHN0cl90b190aXRsZShjb25kaXRpb24pKSkKCnRlc3Rfc3R1ZGllZF9mdWxsIDwtIGZpbHRlcih0ZXN0X2Z1bGwsIHN0dWRpZWQpCmBgYApGaWx0ZXIgb3V0IGV4Y2x1ZGVkIHBhcnRpY2lwYW50OgpgYGB7cn0KczJfZXhjbHVkZSA8LSByZWFkX2ZzdChmaWxlLnBhdGgoIi4uIiwgImRhdGEiLCAicHJvY2Vzc2VkIiwgInMyIiwgInMyX2V4Y2x1ZGUuZnN0IikpCgp0ZXN0IDwtIGFudGlfam9pbih0ZXN0X2Z1bGwsIHMyX2V4Y2x1ZGUsIGJ5ID0gInN1YmplY3QiKSAlPiUgZHJvcGxldmVscygpCnRlc3Rfc3R1ZGllZCA8LSBhbnRpX2pvaW4odGVzdF9zdHVkaWVkX2Z1bGwsIHMyX2V4Y2x1ZGUsIGJ5ID0gInN1YmplY3QiKSAlPiUgZHJvcGxldmVscygpCmBgYAoKIyBUZXN0IGFjY3VyYWN5CgpBY2NvcmRpbmcgdG8gb3VyIGh5cG90aGVzaXMsIHRlc3QgYWNjdXJhY3kgc2hvdWxkIGJlIGhpZ2hlciBpbiB0aGUgKmZhY3QqIGJsb2NrcyB0aGFuIGluIHRoZSAqZGVmYXVsdCogYmxvY2tzLCBzaW5jZSBmYWN0cyBhcmUgcmVwZWF0ZWQgd2l0aCBhIG1vcmUgYXBwcm9wcmlhdGUgc2NoZWR1bGUgZnJvbSB0aGUgc3RhcnQuCgpUaGUgcGxvdCBiZWxvdyBzaG93cyB0aGUgcGVyY2VudGFnZSBvZiBjb3JyZWN0IGFuc3dlcnMgb24gdGhlIHRlc3QgaW4gZWFjaCBibG9jaywgc3BsaXQgYnkgY29uZGl0aW9uIG9yZGVyLgpPbmx5IHRlc3QgaXRlbXMgdGhhdCBhcHBlYXJlZCBkdXJpbmcgdGhlIGxlYXJuaW5nIHNlc3Npb24gYXJlIGluY2x1ZGVkLgoKYGBge3J9CnRlc3RfYWNjX3N0dWRpZWQgPC0gdGVzdF9zdHVkaWVkICU+JQogIGdyb3VwX2J5KHN1YmplY3QsIGJsb2NrLCBjb25kaXRpb24pICU+JQogIHN1bW1hcmlzZShwX2NvcnIgPSBtZWFuKGNvcnJlY3QpKSAlPiUKICAgIG11dGF0ZShjb25kaXRpb25fb3JkZXIgPSBjYXNlX3doZW4oCiAgICBibG9jayA9PSAxICYmIGNvbmRpdGlvbiA9PSAiRGVmYXVsdCIgfiAiRGVmYXVsdC1GYWN0IiwKICAgIGJsb2NrID09IDIgJiYgY29uZGl0aW9uID09ICJGYWN0IiB+ICJEZWZhdWx0LUZhY3QiLAogICAgYmxvY2sgPT0gMSAmJiBjb25kaXRpb24gPT0gIkZhY3QiIH4gIkZhY3QtRGVmYXVsdCIsCiAgICBibG9jayA9PSAyICYmIGNvbmRpdGlvbiA9PSAiRGVmYXVsdCIgfiAiRmFjdC1EZWZhdWx0IgogICkpCmBgYAoKYGBge3J9CmdncGxvdCh0ZXN0X2FjY19zdHVkaWVkLCBhZXMoeCA9IGJsb2NrLCB5ID0gcF9jb3JyKSkgKwogIGZhY2V0X2dyaWQofiBjb25kaXRpb25fb3JkZXIpICsKICBnZW9tX3Zpb2xpbigpICsKICBnZW9tX2JveHBsb3Qod2lkdGggPSAwLjIpICsKICBsYWJzKHggPSAiQmxvY2siLCB5ID0gIkFjY3VyYWN5IikgKwogIHRoZW1lX3Bvc3RlcgpgYGAKCk1ha2UgdGhlIHBsb3Qgc2hvd24gaW4gdGhlIHBhcGVyOgpgYGB7cn0KcCA8LSB0ZXN0X2FjY19zdHVkaWVkICU+JQogIHVuZ3JvdXAoKSAlPiUKICBtdXRhdGUoY29uZGl0aW9uID0gc3RyaW5ncjo6c3RyX3RvX3RpdGxlKGNvbmRpdGlvbikpICU+JQogIGdncGxvdChhZXMoeSA9IHBfY29yciwgeCA9IGNvbmRpdGlvbiwgZ3JvdXAgPSBjb25kaXRpb24pKSArCiAgZ2VvbV9qaXR0ZXIod2lkdGggPSAwLjEsIGhlaWdodCA9IDAsIGFscGhhID0gMC4zLCBhZXMoY29sb3VyID0gY29uZGl0aW9uKSkgKwogIGdlb21fdmlvbGluKGZpbGwgPSBOQSwgd2lkdGggPSAwLjc1KSArCiAgZ2VvbV9ib3hwbG90KHdpZHRoID0gMC4yLCBvdXRsaWVyLnNoYXBlID0gTkEsIGZpbGwgPSBOQSkgKwogIGd1aWRlcyhjb2xvdXIgPSBGQUxTRSkgKwogIGV4cGFuZF9saW1pdHMoeSA9IDApICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50X2Zvcm1hdCgpKSArCiAgc2NhbGVfY29sb3VyX21hbnVhbCh2YWx1ZXMgPSBjb25kaXRpb25fY29sb3VycykgKwogIGxhYnMoeCA9IE5VTEwsCiAgICAgICB5ID0gIlRlc3QgYWNjdXJhY3kiKSArCiAgdGhlbWVfcGFwZXIKCnNhdmVSRFMocCwgIi4uL291dHB1dC90ZXN0LWFjY3VyYWN5LnJkcyIpCgpwCmBgYAoKTWFrZSBwbG90IGZvciBwcmVzZW50YXRpb246CmBgYHtyfQp0aGVtZV9wcmVzZW50YXRpb24gPC0gdGhlbWVfbGlnaHQoYmFzZV9zaXplID0gMTgpICsKICB0aGVtZSh0ZXh0ID0gZWxlbWVudF90ZXh0KGZhbWlseSA9ICJNZXJyaXdlYXRoZXIgU2FucyIsIGNvbG91ciA9ICJibGFjayIpLAogICAgICAgIHN0cmlwLnRleHQgPSBlbGVtZW50X3RleHQoY29sb3VyID0gImJsYWNrIiksCiAgICAgICAgcGxvdC50aXRsZSA9IGVsZW1lbnRfdGV4dChzaXplID0gMTQsIGhqdXN0ID0gMC41KSwKICAgICAgICBheGlzLnRpdGxlID0gZWxlbWVudF90ZXh0KHNpemUgPSAxNCksCiAgICAgICAgcmVjdCA9IGVsZW1lbnRfcmVjdChmaWxsID0gInRyYW5zcGFyZW50IiksCiAgICAgICAgcGxvdC5iYWNrZ3JvdW5kID0gZWxlbWVudF9yZWN0KGZpbGwgPSAidHJhbnNwYXJlbnQiLCBjb2xvciA9IE5BKSwKICAgICAgICBwYW5lbC5iYWNrZ3JvdW5kICA9IGVsZW1lbnRfcmVjdChmaWxsID0gIndoaXRlIiwgY29sb3VyID0gImdyZXk3MCIpCiAgKSAKCnRlc3RfYWNjX3N0dWRpZWQgJT4lCiAgZ2dwbG90KGFlcyh5ID0gcF9jb3JyLCB4ID0gY29uZGl0aW9uLCBncm91cCA9IGNvbmRpdGlvbikpICsKICBnZW9tX2ppdHRlcih3aWR0aCA9IDAuMSwgaGVpZ2h0ID0gMCwgYWxwaGEgPSAwLjMsIGFlcyhjb2xvdXIgPSBjb25kaXRpb24pKSArCiAgZ2VvbV92aW9saW4oZmlsbCA9IE5BLCB3aWR0aCA9IDAuNzUpICsKICBnZW9tX2JveHBsb3Qod2lkdGggPSAwLjIsIG91dGxpZXIuc2hhcGUgPSBOQSwgZmlsbCA9IE5BKSArCiAgZ3VpZGVzKGNvbG91ciA9IEZBTFNFKSArCiAgZXhwYW5kX2xpbWl0cyh5ID0gMCkgKwogIHNjYWxlX3hfZGlzY3JldGUobGFiZWxzID0gYygiQ29sZCBzdGFydCIsICJXYXJtIHN0YXJ0IikpICsKICBzY2FsZV95X2NvbnRpbnVvdXMobGFiZWxzID0gc2NhbGVzOjpwZXJjZW50X2Zvcm1hdCgpKSArCiAgc2NhbGVfY29sb3VyX21hbnVhbCh2YWx1ZXMgPSBjb25kaXRpb25fY29sb3VycykgKwogIGxhYnMoeCA9IE5VTEwsCiAgICAgICB5ID0gIlRlc3QgYWNjdXJhY3kiKSArCiAgdGhlbWVfcHJlc2VudGF0aW9uCgpnZ3NhdmUoIi4uL291dHB1dC90ZXN0LWFjY3VyYWN5LXByZXNlbnRhdGlvbi5wbmciLCBkZXZpY2UgPSAicG5nIiwgd2lkdGggPSA0LCBoZWlnaHQgPSAzLCBiZyA9ICJ0cmFuc3BhcmVudCIpCmBgYAoKCiMgTG9naXN0aWMgbWl4ZWQtZWZmZWN0cyBtb2RlbAoKQXMgcHJlcmVnaXN0ZXJlZCwgd2Ugd2lsbCBldmFsdWF0ZSB0aGUgc3RyZW5ndGggb2YgZXZpZGVuY2UgZm9yIGFuIGVmZmVjdCBvZiBjb25kaXRpb24gb24gdGhlIHByb2JhYmlsaXR5IG9mIGFuc3dlcmluZyB0ZXN0IGl0ZW1zIGNvcnJlY3RseSB1c2luZyBhIGxvZ2lzdGljIG1peGVkIGVmZmVjdHMgbW9kZWwsIHdpdGggbWFpbiBlZmZlY3RzIGZvciBjb25kaXRpb24gYW5kIGJsb2NrLCBhbmQgcmFuZG9tIGludGVyY2VwdHMgZm9yIGl0ZW1zIGFuZCBzdWJqZWN0cy4KVGhlIGZpeGVkIGVmZmVjdHMgd2lsbCBiZSBzdW0tdG8temVybyBjb250cmFzdC1jb2RlZCBhbmQgd2lsbCBoYXZlIENhdWNoeSgwLDEpIHByaW9ycy4KCldlIHdpbGwgdGhlbiBwZXJmb3JtIGEgU2F2YWdlLURpY2tleSBkZW5zaXR5IHJhdGlvIHRlc3QgdG8gZXN0YWJsaXNoIHRoZSBzdHJlbmd0aCBvZiBldmlkZW5jZSBmb3IgYW4gZWZmZWN0IG9mIGNvbmRpdGlvbi4KCiMjIE1vZGVsIGZpdHRpbmcKU2V0IHVwIGNvbnRyYXN0IGNvZGluZyBzbyB0aGF0IHRoZSBtb2RlbCBpbnRlcmNlcHQgcmVmbGVjdHMgdGhlIGdyYW5kIG1lYW4gcChjb3JyZWN0KSwgYWxsb3dpbmcgdXMgdG8gYXNzZXNzIHRoZSBlZmZlY3RzIG9mIGNvbmRpdGlvbiBhbmQgYmxvY2sgc2VwYXJhdGVseToKYGBge3J9CmNvbnRyYXN0cyh0ZXN0X3N0dWRpZWQkY29uZGl0aW9uKSA8LSBjKC0wLjUsIDAuNSkKY29udHJhc3RzKHRlc3Rfc3R1ZGllZCRjb25kaXRpb24pCgpjb250cmFzdHModGVzdF9zdHVkaWVkJGJsb2NrKSA8LSBjKC0wLjUsIDAuNSkKY29udHJhc3RzKHRlc3Rfc3R1ZGllZCRibG9jaykKYGBgCgpTZXQgdGhlIHByaW9ycyBmb3IgdGhlIGZpeGVkIGVmZmVjdHM6CmBgYHtyfQpwcmlvcl9tMSA8LSBzZXRfcHJpb3IoImNhdWNoeSgwLCAxKSIsIGNsYXNzID0gImIiKQpgYGAKCkZpdCB0aGUgbW9kZWw6CmBgYHtyfQptMSA8LSBicm0oCiAgY29ycmVjdCB+IGNvbmRpdGlvbiArIGJsb2NrICsgKDEgfCBzdWJqZWN0KSArICgxIHwgZmFjdF9pZCksCiAgZmFtaWx5ID0gYmVybm91bGxpLAogIGRhdGEgPSB0ZXN0X3N0dWRpZWQsCiAgcHJpb3IgPSBwcmlvcl9tMSwKICBjaGFpbnMgPSA0LAogIGl0ZXIgPSAxMDAwMCwKICBzYXZlX2FsbF9wYXJzID0gVFJVRSwKICBzYW1wbGVfcHJpb3IgPSBUUlVFLAogIGZ1dHVyZSA9IFRSVUUsCiAgc2VlZCA9IDAsCiAgZmlsZSA9ICJtb2RlbF9maXRzL20xIgopCmBgYAoKIyMgTW9kZWwgaW5zcGVjdGlvbgpFbnN1cmUgdGhhdCBNQ01DIHdlbnQgYXMgZXhwZWN0ZWQuClRoZSBjYXRlcnBpbGxhciBwbG90cyBsb29rIG5vcm1hbCwgYW5kIHRoZXJlIGlzIG5vIGV2aWRlbmNlIG9mIGF1dG9jb3JyZWxhdGlvbiBpbiB0aGUgY2hhaW5zLgpgYGB7ciwgZmlnLndpZHRoID0gMTJ9Cm1jbWNfcGxvdChtMSwgcGFycyA9IGMoImNvbmRpdGlvbiIsICJibG9jayIpLCB0eXBlID0gInRyYWNlIikKCm1jbWNfcGxvdChtMSwgcGFycyA9IGMoImNvbmRpdGlvbiIsICJibG9jayIpLCB0eXBlID0gImFjZl9iYXIiKQpgYGAKCiMjIE1vZGVsIGludGVycHJldGF0aW9uCk1vZGVsIHN1bW1hcnk6CmBgYHtyfQpzdW1tYXJ5KG0xKQpgYGAKClZpc3VhbGlzZSB0aGUgbW9kZWwncyBwb3B1bGF0aW9uLWxldmVsIGVzdGltYXRlcywgYWxvbmcgd2l0aCB0aGVpciA5NSUgY3JlZGlibGUgaW50ZXJ2YWxzOgpgYGB7cn0KbWNtY19wbG90KG0xLCBwYXJzID0gYygiY29uZGl0aW9uIiwgImJsb2NrIiksIHR5cGUgPSAiYXJlYXMiLCBwcm9iID0gLjk1KQpgYGAKClRoZSBtb2RlbCBtYWtlcyB0aGUgZm9sbG93aW5nIHByZWRpY3Rpb25zLCBiYXNlZCBvbiBpdHMgZml4ZWQgZWZmZWN0cy4KCmBgYHtyfQojIEdldCBmaXR0ZWQgdmFsdWVzCm5ld19kYXRhIDwtIGNyb3NzaW5nKGNvbmRpdGlvbiA9IGxldmVscyh0ZXN0X3N0dWRpZWQkY29uZGl0aW9uKSwgYmxvY2sgPSBsZXZlbHModGVzdF9zdHVkaWVkJGJsb2NrKSkKZml0X20xIDwtIGRhdGEudGFibGUoZml0dGVkKG0xLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgbmV3ZGF0YSA9IG5ld19kYXRhLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgcmVfZm9ybXVsYSA9IE5BLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgc3VtbWFyeSA9IEZBTFNFKSkKc2V0bmFtZXMoZml0X20xLCBhcy5jaGFyYWN0ZXIoaW50ZXJhY3Rpb24obmV3X2RhdGEkY29uZGl0aW9uLCBuZXdfZGF0YSRibG9jaykpKQoKZml0X20xX3N1bW1hcnkgPC0gZGF0YS50YWJsZShmaXR0ZWQobTEsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5ld2RhdGEgPSBuZXdfZGF0YSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcmVfZm9ybXVsYSA9IE5BKSkKc2V0bmFtZXMoZml0X20xX3N1bW1hcnksIGMoImZpdCIsICJzZSIsICJsb3dlciIsICJ1cHBlciIpKQpmaXRfbTFfc3VtbWFyeSA8LSBjYmluZChuZXdfZGF0YSwgZml0X20xX3N1bW1hcnkpCmBgYAoKCk92ZXJhbGwgcmVzcG9uc2UgYWNjdXJhY3kgKGFjcm9zcyBibG9ja3MgYW5kIGNvbmRpdGlvbnMpOgpgYGB7cn0KcXVhbnRpbGUoKGZpdF9tMSREZWZhdWx0LjEgKyBmaXRfbTEkRGVmYXVsdC4yICsgZml0X20xJEZhY3QuMSArIGZpdF9tMSRGYWN0LjIpLzQsCiAgICAgICAgIHByb2JzID0gYyguNSwgLjAyNSwgLjk3NSkpCmBgYAoKRGlmZmVyZW5jZSBpbiByZXNwb25zZSBhY2N1cmFjeSBiZXR3ZWVuIEZhY3QgY29uZGl0aW9uIGFuZCBEZWZhdWx0IGNvbmRpdGlvbiAoYWNyb3NzIGJsb2Nrcyk6IApgYGB7cn0KZmFjdF92c19kZWZhdWx0IDwtIChmaXRfbTEkRmFjdC4xICsgZml0X20xJEZhY3QuMikvMiAtIChmaXRfbTEkRGVmYXVsdC4xICsgZml0X20xJERlZmF1bHQuMikvMgpxdWFudGlsZShmYWN0X3ZzX2RlZmF1bHQsIHByb2JzID0gYyguNSwgLjAyNSwgLjk3NSkpCmBgYAoKRGlmZmVyZW5jZSBpbiByZXNwb25zZSBhY2N1cmFjeSBiZXR3ZWVuIGJsb2NrIDIgYW5kIGJsb2NrIDEgKGFjcm9zcyBjb25kaXRpb25zKToKYGBge3J9CmJsb2NrMl92c19ibG9jazEgPC0gKGZpdF9tMSREZWZhdWx0LjIgKyBmaXRfbTEkRmFjdC4yKS8yIC0gKGZpdF9tMSREZWZhdWx0LjEgKyBmaXRfbTEkRmFjdC4xKS8yCnF1YW50aWxlKGJsb2NrMl92c19ibG9jazEsIHByb2JzID0gYyguNSwgLjAyNSwgLjk3NSkpCmBgYAoKCgojIyBIeXBvdGhlc2lzIHRlc3RpbmcKCk5vdyB3ZSBjYW4gdGVzdCB0aGUgaHlwb3RoZXNpcyB0aGF0IHRoZXJlIGlzIGFuIGVmZmVjdCBvZiBjb25kaXRpb24gYWdhaW5zdCB0aGUgbnVsbCBoeXBvdGhlc2lzIHRoYXQgdGhlcmUgaXMgbm90LgpXZSBkbyB0aGlzIGJ5IGNvbXB1dGluZyB0aGUgZGVuc2l0eSByYXRpbyBiZXR3ZWVuIHRoZSBwb3N0ZXJpb3IgYW5kIHRoZSBwcmlvciBvZiB0aGUgY29uZGl0aW9uIGNvZWZmaWNpZW50IGF0IDAuClRoZSBwcmlvciBpcyBhIENhdWNoeSgwLDEpIGRpc3RyaWJ1dGlvbiwgd2hpY2ggbWVhbnMgd2UgY2FuIGNhbGN1bGF0ZSBpdHMgZGVuc2l0eSBhdCAwIGFuYWx5dGljYWxseS4KVGhlIHBvc3RlcmlvciBkZW5zaXR5IGlzIGVzdGltYXRlZCBvbiB0aGUgYmFzaXMgb2YgdGhlIHBvc3RlcmlvciBzYW1wbGVzIGZyb20gdGhlIG1vZGVsLgoKYGBge3J9CnByaW9yX2RlbnNpdHlfYXRfemVybyA8LSBkY2F1Y2h5KDAsIDAsIDEpCm0xX3Bvc3Rlcmlvcl9zYW1wbGVzIDwtIHBvc3Rlcmlvcl9zYW1wbGVzKG0xLCBwYXJzID0gImJfY29uZGl0aW9uMSIpJGJfY29uZGl0aW9uMQpwb3N0ZXJpb3JfZGVuc2l0eV9hdF96ZXJvIDwtIGRlbnNpdHlfcmF0aW8obTFfcG9zdGVyaW9yX3NhbXBsZXMsIHBvaW50ID0gMCkKCgp4dmFsIDwtIHNlcSgtMiwgMiwgYnkgPSAwLjAxKQpwbG90X3ByaW9yIDwtIGRjYXVjaHkoeHZhbCwgbG9jYXRpb24gPSAwLCBzY2FsZSA9IDEpCnBsb3RfcG9zdGVyaW9yIDwtIGRlbnNpdHlfcmF0aW8oeCA9IG0xX3Bvc3Rlcmlvcl9zYW1wbGVzLCB5ID0gTlVMTCwgcG9pbnQgPSB4dmFsKSAjIFJldHVybnMgdGhlIGRlbnNpdHkgb2YgdGhlIHBvc3RlcmlvciBhdCBlYWNoIHBvaW50IGluIHgKbTFfZGVuc2l0eSA8LSB0aWJibGUoeHZhbCA9IHh2YWwsIFByaW9yID0gcGxvdF9wcmlvciwgUG9zdGVyaW9yID0gcGxvdF9wb3N0ZXJpb3IpCmBgYAoKYGBge3J9CmdhdGhlcihtMV9kZW5zaXR5LCAteHZhbCwga2V5ID0gIlR5cGUiLCB2YWx1ZSA9ICJkZW5zaXR5IikgJT4lCiAgbXV0YXRlKGRlbnNpdHkgPSBpZmVsc2UoZGVuc2l0eSA8IDAsIDAsIGRlbnNpdHkpKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSB4dmFsLCB5ID0gZGVuc2l0eSwgZmlsbCA9IFR5cGUpKSArCiAgZ2VvbV9hcmVhKHBvc2l0aW9uID0gImlkZW50aXR5IiwgY29sb3VyID0gImJsYWNrIiwgYWxwaGEgPSAwLjc1KSArCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgbHR5ID0gMykgKwogIGFubm90YXRlKCJwb2ludCIsIHggPSBjKDAsIDApLCB5ID0gYyhwcmlvcl9kZW5zaXR5X2F0X3plcm8sIHBvc3Rlcmlvcl9kZW5zaXR5X2F0X3plcm8pKSArCiAgbGFicyh4ID0gIkVzdGltYXRlIGZvciBjb2VmZmljaWVudCAnY29uZGl0aW9uJyIsIHkgPSBOVUxMLCB0aXRsZSA9ICIiKSArCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gYygiIzAzMzk2YyIsICIjYjNjZGUwIikpCgpnZ3NhdmUoZmlsZS5wYXRoKCIuLiIsICJvdXRwdXQiLCAic2RfcmF0aW8ucGRmIiksIGRldmljZSA9ICJwZGYiLCB3aWR0aCA9IDQsIGhlaWdodCA9IDMpCmBgYAoKVGhlIGV2aWRlbmNlIGluIGZhdm91ciBvZiBhIG5vbi16ZXJvIGNvZWZmaWNpZW50IGZvciBjb25kaXRpb24gaXMgZXZhbHVhdGVkIGJ5IG1lYW5zIG9mIHRoZSBTYXZhZ2UtRGlja2V5IGRlbnNpdHkgcmF0aW86IHRoZSByYXRpbyBiZXR3ZWVuIHRoZSBwcmlvciBhbmQgcG9zdGVyaW9yIGRlbnNpdHkgYXQgJFxiZXRhX3tjb25kaXRpb259ID0gMCQgKHRoZSB0d28gcG9pbnRzIG9uIHRoZSBkb3R0ZWQgbGluZSBpbiB0aGUgcGxvdCBhYm92ZSkuCmBgYHtyfQpCRl9jb25kaXRpb24gPC0gcHJpb3JfZGVuc2l0eV9hdF96ZXJvIC8gcG9zdGVyaW9yX2RlbnNpdHlfYXRfemVybwoKQkZfY29uZGl0aW9uCmBgYAoKVGhlIGV2aWRlbmNlIGluIGZhdm91ciBvZiBhbiBlZmZlY3Qgb2YgY29uZGl0aW9uIGlzICRCRl97MTB9JCA9IGByIGZvcm1hdChCRl9jb25kaXRpb24pYC4KCgoKIyBTYW5pdHkgY2hlY2tzCgojIyBUZXN0IGVmZmVjdCBvZiBwcmlvcgoKV2hhdCBpZiB3ZSBjaG9vc2UgYSB3ZWFrZXIgQ2F1Y2h5KDAsIDEwKSBwcmlvcj8KYGBge3J9CnByaW9yX20xX3dlYWsgPC0gc2V0X3ByaW9yKCJjYXVjaHkoMCwgMTApIiwgY2xhc3MgPSAiYiIpCmBgYAoKRml0IHRoZSBtb2RlbDoKYGBge3J9Cm0xX3dlYWsgPC0gYnJtKAogIGNvcnJlY3QgfiBjb25kaXRpb24gKyBibG9jayArICgxIHwgc3ViamVjdCkgKyAoMSB8IGZhY3RfaWQpLAogIGZhbWlseSA9IGJlcm5vdWxsaSwKICBkYXRhID0gdGVzdF9zdHVkaWVkLAogIHByaW9yID0gcHJpb3JfbTFfd2VhaywKICBjaGFpbnMgPSA0LAogIGl0ZXIgPSAxMDAwMCwKICBzYXZlX2FsbF9wYXJzID0gVFJVRSwKICBzYW1wbGVfcHJpb3IgPSBUUlVFLAogIGZ1dHVyZSA9IFRSVUUsCiAgc2VlZCA9IDAsCiAgZmlsZSA9ICJtb2RlbF9maXRzL20xX3dlYWsiCikKYGBgCgoKYGBge3J9CnByaW9yX2RlbnNpdHlfYXRfemVyb193ZWFrIDwtIGRjYXVjaHkoMCwgMCwgMTApCm0xX3dlYWtfcG9zdGVyaW9yX3NhbXBsZXMgPC0gcG9zdGVyaW9yX3NhbXBsZXMobTFfd2VhaywgcGFycyA9ICJiX2NvbmRpdGlvbjEiKSRiX2NvbmRpdGlvbjEKcG9zdGVyaW9yX2RlbnNpdHlfYXRfemVyb193ZWFrIDwtIGRlbnNpdHlfcmF0aW8obTFfd2Vha19wb3N0ZXJpb3Jfc2FtcGxlcywgcG9pbnQgPSAwKQoKCnh2YWwgPC0gc2VxKC0yLCAyLCBieSA9IDAuMDEpCnBsb3RfcHJpb3Jfd2VhayA8LSBkY2F1Y2h5KHh2YWwsIGxvY2F0aW9uID0gMCwgc2NhbGUgPSAxMCkKcGxvdF9wb3N0ZXJpb3Jfd2VhayA8LSBkZW5zaXR5X3JhdGlvKHggPSBtMV93ZWFrX3Bvc3Rlcmlvcl9zYW1wbGVzLCB5ID0gTlVMTCwgcG9pbnQgPSB4dmFsKSAjIFJldHVybnMgdGhlIGRlbnNpdHkgb2YgdGhlIHBvc3RlcmlvciBhdCBlYWNoIHBvaW50IGluIHgKbTFfd2Vha19kZW5zaXR5IDwtIHRpYmJsZSh4dmFsID0geHZhbCwgUHJpb3IgPSBwbG90X3ByaW9yX3dlYWssIFBvc3RlcmlvciA9IHBsb3RfcG9zdGVyaW9yX3dlYWspCmBgYAoKYGBge3J9CmdhdGhlcihtMV93ZWFrX2RlbnNpdHksIC14dmFsLCBrZXkgPSAiVHlwZSIsIHZhbHVlID0gImRlbnNpdHkiKSAlPiUKICBtdXRhdGUoZGVuc2l0eSA9IGlmZWxzZShkZW5zaXR5IDwgMCwgMCwgZGVuc2l0eSkpICU+JQogIGdncGxvdChhZXMoeCA9IHh2YWwsIHkgPSBkZW5zaXR5LCBmaWxsID0gVHlwZSkpICsKICBnZW9tX2FyZWEocG9zaXRpb24gPSAiaWRlbnRpdHkiLCBjb2xvdXIgPSAiYmxhY2siLCBhbHBoYSA9IDAuNzUpICsKICBnZW9tX3ZsaW5lKHhpbnRlcmNlcHQgPSAwLCBsdHkgPSAzKSArCiAgYW5ub3RhdGUoInBvaW50IiwgeCA9IGMoMCwgMCksIHkgPSBjKHByaW9yX2RlbnNpdHlfYXRfemVyb193ZWFrLCBwb3N0ZXJpb3JfZGVuc2l0eV9hdF96ZXJvX3dlYWspKSArCiAgbGFicyh4ID0gIkVzdGltYXRlIGZvciBjb2VmZmljaWVudCAnY29uZGl0aW9uJyIsIHkgPSBOVUxMLCB0aXRsZSA9ICIiKSArCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gYygiIzAzMzk2YyIsICIjYjNjZGUwIikpCmBgYAoKRXZlbiB3aXRoIHRoZSBtdWNoIHdlYWtlciBwcmlvciwgdGhlIGV2aWRlbmNlIGluIGZhdm91ciBvZiBhbiBlZmZlY3Qgb2YgY29uZGl0aW9uIGlzIHN0aWxsIHZlcnkgc3Ryb25nLgoKYGBge3J9CkJGX2NvbmRpdGlvbl93ZWFrIDwtIHByaW9yX2RlbnNpdHlfYXRfemVyb193ZWFrIC8gcG9zdGVyaW9yX2RlbnNpdHlfYXRfemVyb193ZWFrCgpCRl9jb25kaXRpb25fd2VhawpgYGAKCkhvdyBhYm91dCBhIHN0cm9uZ2VyIENhdWNoeSgwLCAwLjUpIHByaW9yPwpgYGB7cn0KcHJpb3JfbTFfc3Ryb25nIDwtIHNldF9wcmlvcigiY2F1Y2h5KDAsIDAuNSkiLCBjbGFzcyA9ICJiIikKYGBgCgpGaXQgdGhlIG1vZGVsOgpgYGB7cn0KbTFfc3Ryb25nIDwtIGJybSgKICBjb3JyZWN0IH4gY29uZGl0aW9uICsgYmxvY2sgKyAoMSB8IHN1YmplY3QpICsgKDEgfCBmYWN0X2lkKSwKICBmYW1pbHkgPSBiZXJub3VsbGksCiAgZGF0YSA9IHRlc3Rfc3R1ZGllZCwKICBwcmlvciA9IHByaW9yX20xX3N0cm9uZywKICBjaGFpbnMgPSA0LAogIGl0ZXIgPSAxMDAwMCwKICBzYXZlX2FsbF9wYXJzID0gVFJVRSwKICBzYW1wbGVfcHJpb3IgPSBUUlVFLAogIGZ1dHVyZSA9IFRSVUUsCiAgc2VlZCA9IDAsCiAgZmlsZSA9ICJtb2RlbF9maXRzL20xX3N0cm9uZyIKKQpgYGAKCgpgYGB7cn0KcHJpb3JfZGVuc2l0eV9hdF96ZXJvX3N0cm9uZyA8LSBkY2F1Y2h5KDAsIDAsIDAuNSkKbTFfc3Ryb25nX3Bvc3Rlcmlvcl9zYW1wbGVzIDwtIHBvc3Rlcmlvcl9zYW1wbGVzKG0xX3N0cm9uZywgcGFycyA9ICJiX2NvbmRpdGlvbjEiKSRiX2NvbmRpdGlvbjEKcG9zdGVyaW9yX2RlbnNpdHlfYXRfemVyb19zdHJvbmcgPC0gZGVuc2l0eV9yYXRpbyhtMV9zdHJvbmdfcG9zdGVyaW9yX3NhbXBsZXMsIHBvaW50ID0gMCkKCgp4dmFsIDwtIHNlcSgtMiwgMiwgYnkgPSAwLjAxKQpwbG90X3ByaW9yX3N0cm9uZyA8LSBkY2F1Y2h5KHh2YWwsIGxvY2F0aW9uID0gMCwgc2NhbGUgPSAwLjUpCnBsb3RfcG9zdGVyaW9yX3N0cm9uZyA8LSBkZW5zaXR5X3JhdGlvKHggPSBtMV9zdHJvbmdfcG9zdGVyaW9yX3NhbXBsZXMsIHkgPSBOVUxMLCBwb2ludCA9IHh2YWwpICMgUmV0dXJucyB0aGUgZGVuc2l0eSBvZiB0aGUgcG9zdGVyaW9yIGF0IGVhY2ggcG9pbnQgaW4geAptMV9zdHJvbmdfZGVuc2l0eSA8LSB0aWJibGUoeHZhbCA9IHh2YWwsIFByaW9yID0gcGxvdF9wcmlvcl9zdHJvbmcsIFBvc3RlcmlvciA9IHBsb3RfcG9zdGVyaW9yX3N0cm9uZykKYGBgCgpgYGB7cn0KZ2F0aGVyKG0xX3N0cm9uZ19kZW5zaXR5LCAteHZhbCwga2V5ID0gIlR5cGUiLCB2YWx1ZSA9ICJkZW5zaXR5IikgJT4lCiAgbXV0YXRlKGRlbnNpdHkgPSBpZmVsc2UoZGVuc2l0eSA8IDAsIDAsIGRlbnNpdHkpKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSB4dmFsLCB5ID0gZGVuc2l0eSwgZmlsbCA9IFR5cGUpKSArCiAgZ2VvbV9hcmVhKHBvc2l0aW9uID0gImlkZW50aXR5IiwgY29sb3VyID0gImJsYWNrIiwgYWxwaGEgPSAwLjc1KSArCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgbHR5ID0gMykgKwogIGFubm90YXRlKCJwb2ludCIsIHggPSBjKDAsIDApLCB5ID0gYyhwcmlvcl9kZW5zaXR5X2F0X3plcm9fc3Ryb25nLCBwb3N0ZXJpb3JfZGVuc2l0eV9hdF96ZXJvX3N0cm9uZykpICsKICBsYWJzKHggPSAiRXN0aW1hdGUgZm9yIGNvZWZmaWNpZW50ICdjb25kaXRpb24nIiwgeSA9IE5VTEwsIHRpdGxlID0gIiIpICsKICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjKCIjMDMzOTZjIiwgIiNiM2NkZTAiKSkKYGBgCgpXaXRoIHRoZSBzdHJvbmdlciBwcmlvciwgdGhlIGV2aWRlbmNlIGluIGZhdm91ciBvZiBhbiBlZmZlY3Qgb2YgY29uZGl0aW9uIGlzIGFnYWluIHZlcnkgc3Ryb25nLgoKYGBge3J9CkJGX2NvbmRpdGlvbl9zdHJvbmcgPC0gcHJpb3JfZGVuc2l0eV9hdF96ZXJvX3N0cm9uZyAvIHBvc3Rlcmlvcl9kZW5zaXR5X2F0X3plcm9fc3Ryb25nCgpCRl9jb25kaXRpb25fc3Ryb25nCmBgYAoKCgojIyBUZXN0IGVmZmVjdCBvZiBvdXRsaWVycwoKVGhlIGFuYWx5c2lzIGFib3ZlIGRpZCBub3QgaW5jbHVkZSBkYXRhIGZyb20gZXhjbHVkZWQgcGFydGljaXBhbnRzIChzZWUgUHJlcGFyZSBkYXRhIHNlY3Rpb24pLgpWZXJpZnkgdGhhdCB0aGlzIGRvZXMgbm90IGFmZmVjdCB0aGUgY29uY2x1c2lvbiBieSBhbHNvIGZpdHRpbmcgdGhlIG1vZGVsIHRvIHRoZSBmdWxsIGRhdGFzZXQuCgpgYGB7cn0KY29udHJhc3RzKHRlc3Rfc3R1ZGllZF9mdWxsJGNvbmRpdGlvbikgPC0gYygtMC41LCAwLjUpCmNvbnRyYXN0cyh0ZXN0X3N0dWRpZWRfZnVsbCRjb25kaXRpb24pCgpjb250cmFzdHModGVzdF9zdHVkaWVkX2Z1bGwkYmxvY2spIDwtIGMoLTAuNSwgMC41KQpjb250cmFzdHModGVzdF9zdHVkaWVkX2Z1bGwkYmxvY2spCmBgYAoKCmBgYHtyfQptMV9mdWxsIDwtIGJybSgKICBjb3JyZWN0IH4gY29uZGl0aW9uICsgYmxvY2sgKyAoMSB8IHN1YmplY3QpICsgKDEgfCBmYWN0X2lkKSwKICBmYW1pbHkgPSBiZXJub3VsbGksCiAgZGF0YSA9IHRlc3Rfc3R1ZGllZF9mdWxsLAogIHByaW9yID0gcHJpb3JfbTEsCiAgY2hhaW5zID0gNCwKICBpdGVyID0gMTAwMDAsCiAgc2F2ZV9hbGxfcGFycyA9IFRSVUUsCiAgc2FtcGxlX3ByaW9yID0gVFJVRSwKICBmdXR1cmUgPSBUUlVFLAogIHNlZWQgPSAwLAogIGZpbGUgPSAibW9kZWxfZml0cy9tMV9mdWxsIgopCmBgYAoKQ2hlY2sgdGhlIGNoYWluczoKYGBge3J9Cm1jbWNfcGxvdChtMV9mdWxsLCBwYXJzID0gYygiY29uZGl0aW9uIiwgImJsb2NrIiksIHR5cGUgPSAidHJhY2UiKQptY21jX3Bsb3QobTFfZnVsbCwgcGFycyA9IGMoImNvbmRpdGlvbiIsICJibG9jayIpLCB0eXBlID0gImFjZl9iYXIiKQpgYGAKClRoZSBtb2RlbCBlc3RpbWF0ZXMgbG9vayB2ZXJ5IHNpbWlsYXIuCmBgYHtyfQpzdW1tYXJ5KG0xX2Z1bGwpCmBgYAoKYGBge3J9Cm0xX2Z1bGxfcG9zdGVyaW9yX3NhbXBsZXMgPC0gcG9zdGVyaW9yX3NhbXBsZXMobTFfZnVsbCwgcGFycyA9ICJiX2NvbmRpdGlvbjEiKSRiX2NvbmRpdGlvbjEKcG9zdGVyaW9yX2RlbnNpdHlfYXRfemVyb19mdWxsIDwtIGRlbnNpdHlfcmF0aW8obTFfZnVsbF9wb3N0ZXJpb3Jfc2FtcGxlcywgcG9pbnQgPSAwKQoKcGxvdF9wb3N0ZXJpb3JfZnVsbCA8LSBkZW5zaXR5X3JhdGlvKHggPSBtMV9mdWxsX3Bvc3Rlcmlvcl9zYW1wbGVzLCB5ID0gTlVMTCwgcG9pbnQgPSB4dmFsKSAjIFJldHVybnMgdGhlIGRlbnNpdHkgb2YgdGhlIHBvc3RlcmlvciBhdCBlYWNoIHBvaW50IGluIHgKbTFfZnVsbF9kZW5zaXR5IDwtIHRpYmJsZSh4dmFsID0geHZhbCwgUHJpb3IgPSBwbG90X3ByaW9yLCBQb3N0ZXJpb3IgPSBwbG90X3Bvc3Rlcmlvcl9mdWxsKQpgYGAKCmBgYHtyfQpnYXRoZXIobTFfZnVsbF9kZW5zaXR5LCAteHZhbCwga2V5ID0gIlR5cGUiLCB2YWx1ZSA9ICJkZW5zaXR5IikgJT4lCiAgbXV0YXRlKGRlbnNpdHkgPSBpZmVsc2UoZGVuc2l0eSA8IDAsIDAsIGRlbnNpdHkpKSAlPiUKICBnZ3Bsb3QoYWVzKHggPSB4dmFsLCB5ID0gZGVuc2l0eSwgZmlsbCA9IFR5cGUpKSArCiAgZ2VvbV9hcmVhKHBvc2l0aW9uID0gImlkZW50aXR5IiwgY29sb3VyID0gImJsYWNrIiwgYWxwaGEgPSAwLjc1KSArCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gMCwgbHR5ID0gMykgKwogIGFubm90YXRlKCJwb2ludCIsIHggPSBjKDAsIDApLCB5ID0gYyhwcmlvcl9kZW5zaXR5X2F0X3plcm8sIHBvc3Rlcmlvcl9kZW5zaXR5X2F0X3plcm8pKSArCiAgbGFicyh4ID0gIkVzdGltYXRlIGZvciBjb2VmZmljaWVudCAnY29uZGl0aW9uJyIsIHkgPSBOVUxMLCB0aXRsZSA9ICIiKSArCiAgc2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gYygiIzAzMzk2YyIsICIjYjNjZGUwIikpCmBgYAoKVGhlIFNhdmFnZS1EaWNrZXkgZGVuc2l0eSByYXRpbyBpcyB2ZXJ5IHNpbWlsYXIuCmBgYHtyfQpCRl9jb25kaXRpb25fZnVsbCA8LSBwcmlvcl9kZW5zaXR5X2F0X3plcm8gLyBwb3N0ZXJpb3JfZGVuc2l0eV9hdF96ZXJvX2Z1bGwKCkJGX2NvbmRpdGlvbl9mdWxsCmBgYAoKCgojIyBDb21wYXJlIHRvIGZyZXF1ZW50aXN0IG1vZGVsCgpGaXQgYSBmcmVxdWVudGlzdCBnbG1lciB3aXRoIHRoZSBzYW1lIG1vZGVsIHN0cnVjdHVyZS4KVGhlIG1vZGVsIHN1bW1hcnkgY29uZmlybXMgdGhhdCB3ZSBnZXQgdGhlIHNhbWUgY29lZmZpY2llbnQgZXN0aW1hdGVzIGFzIHdpdGggdGhlIEJheWVzaWFuIG1vZGVsLgpgYGB7cn0KbTFfZnJlcSA8LSBnbG1lcihjb3JyZWN0IH4gY29uZGl0aW9uICsgYmxvY2sgKyAoMSB8IHN1YmplY3QpICsgKDEgfCBmYWN0X2lkKSwKICAgICAgICAgICAgICAgICBmYW1pbHkgPSBiaW5vbWlhbCwKICAgICAgICAgICAgICAgICBkYXRhID0gdGVzdF9zdHVkaWVkKQoKc3VtbWFyeShtMV9mcmVxKQpgYGAKCgojIyBDb21wYXJlIFNEIGRlbnNpdHkgcmF0aW8gdG8gYnJpZGdlIHNhbXBsaW5nCgpBbiBhbHRlcm5hdGl2ZSBtZXRob2QgdG8gdGVzdCBmb3IgYW4gZWZmZWN0IG9mIGNvbmRpdGlvbiBpcyB0byBjb21wYXJlIHRoZSBtYXJnaW5hbCBsaWtlbGlob29kIG9mIGEgbW9kZWwgd2l0aCB0aGUgcHJlZGljdG9yIHRvIHRoYXQgb2YgYSBtb2RlbCB3aXRob3V0IGl0LgpXZSBjYW4gb2J0YWluIHRoZSBtYXJnaW5hbCBsaWtlbGlob29kcyB1c2luZyBicmlkZ2Ugc2FtcGxpbmcuClRoaXMgY29tcGFyaXNvbiB5aWVsZHMgYSBCYXllcyBmYWN0b3IgdGhhdCBjYW4gYmUgaW50ZXJwcmV0ZWQgaW4gdGhlIHNhbWUgd2F5LgoKRml0IGEgc2Vjb25kIG1vZGVsIHdpdGhvdXQgY29uZGl0aW9uIGFzIGEgcHJlZGljdG9yOgpgYGB7cn0KbTIgPC0gYnJtKAogIGNvcnJlY3QgfiBibG9jayArICgxIHwgc3ViamVjdCkgKyAoMSB8IGZhY3RfaWQpLAogIGZhbWlseSA9IGJlcm5vdWxsaSwKICBkYXRhID0gdGVzdF9zdHVkaWVkLAogIHByaW9yID0gcHJpb3JfbTEsCiAgY2hhaW5zID0gNCwKICBpdGVyID0gMTAwMDAsCiAgc2F2ZV9hbGxfcGFycyA9IFRSVUUsCiAgc2FtcGxlX3ByaW9yID0gVFJVRSwKICBmdXR1cmUgPSBUUlVFLAogIHNlZWQgPSAwLAogIGZpbGUgPSAibW9kZWxfZml0cy9tMiIKKQpgYGAKCkNhbGN1bGF0ZSB0aGUgQmF5ZXMgZmFjdG9yOgpgYGB7cn0KYmF5ZXNfZmFjdG9yKG0xLCBtMikKYGBgCgpBZ2FpbiwgaXQncyBiYXNpY2FsbHkgdGhlIHNhbWUgYXMgYmVmb3JlLgoKCgojIEFjY3VtdWxhdGlvbiBvZiBldmlkZW5jZQoKSG93IGRvZXMgJEJGX3sxMH0kIGRldmVsb3AgYXMgbW9yZSBwYXJ0aWNpcGFudHMgYXJlIGFkZGVkPwpUaGUgZm9sbG93aW5nIGNvZGUgZml0cyB0aGUgbW9kZWwgcmVwZWF0ZWRseSwgZWFjaCB0aW1lIGFkZGluZyBzZXZlcmFsIG1vcmUgcGFydGljaXBhbnRzLCBhbmQgY2FsY3VsYXRlcyB0aGUgJEJGX3sxMH0kIGZvciBlYWNoIGZpdC4KCmBgYHtyfQpzdWJqZWN0cyA8LSB1bmlxdWUodGVzdF9zdHVkaWVkJHN1YmplY3QpCnNldF9zaXplcyA8LSBzZXEoMSwgbGVuZ3RoKHN1YmplY3RzKSwgYnkgPSAxKQoKZXZpZGVuY2UgPC0gdGliYmxlKG4gPSAwLCBiZiA9IE5BLCBwb3N0ZXJpb3IgPSBsaXN0KHJjYXVjaHkobiA9IDIwMDAwLCBsb2NhdGlvbiA9IDAsIHNjYWxlID0gMSkpKQoKZm9yIChpIGluIHNldF9zaXplcykgewogIAogICMgU3Vic2V0IHRoZSBkYXRhCiAgdGVzdF9zdHVkaWVkX3N1YnNldCA8LSBmaWx0ZXIodGVzdF9zdHVkaWVkLCBzdWJqZWN0ICVpbiUgc3ViamVjdHNbMTppXSkKICAKICAjIEZpdCB0aGUgbW9kZWwKICBtMV9zZXEgPC0gYnJtKGNvcnJlY3QgfiBjb25kaXRpb24gKyBibG9jayArICgxIHwgc3ViamVjdCkgKyAoMSB8IGZhY3RfaWQpLCAKICAgICAgICAgICBmYW1pbHkgPSBiZXJub3VsbGksCiAgICAgICAgICAgZGF0YSA9IHRlc3Rfc3R1ZGllZF9zdWJzZXQsCiAgICAgICAgICAgcHJpb3IgPSBwcmlvcl9tMSwKICAgICAgICAgICBjaGFpbnMgPSA0LAogICAgICAgICAgIGl0ZXIgPSAxMDAwMCwKICAgICAgICAgICBzYXZlX2FsbF9wYXJzID0gVFJVRSwKICAgICAgICAgICBzYW1wbGVfcHJpb3IgPSBUUlVFLAogICAgICAgICAgIGZ1dHVyZSA9IFRSVUUsCiAgICAgICAgICAgcmVmcmVzaCA9IDAsCiAgICAgICAgICAgb3Blbl9wcm9ncmVzcyA9IEZBTFNFLAogICAgICAgICAgIHNlZWQgPSAwLAogICAgICAgICAgIGZpbGUgPSBwYXN0ZTAoIm1vZGVsX2ZpdHMvbTFfc2VxIiwgaSkpCiAgCiAgbTFfc2VxX3Bvc3Rlcmlvcl9zYW1wbGVzIDwtIHBvc3Rlcmlvcl9zYW1wbGVzKG0xX3NlcSwgcGFycyA9ICJiX2NvbmRpdGlvbjEiKSRiX2NvbmRpdGlvbjEKICBwb3N0ZXJpb3JfZGVuc2l0eV9hdF96ZXJvX3NlcSA8LSBkZW5zaXR5X3JhdGlvKG0xX3NlcV9wb3N0ZXJpb3Jfc2FtcGxlcywgcG9pbnQgPSAwKQoKICBCRl9jb25kaXRpb25fc2VxIDwtIHByaW9yX2RlbnNpdHlfYXRfemVybyAvIHBvc3Rlcmlvcl9kZW5zaXR5X2F0X3plcm9fc2VxCiAgCiAgZXZpZGVuY2Vfc2VxIDwtIHRpYmJsZShuID0gaSwgYmYgPSBCRl9jb25kaXRpb25fc2VxLCBwb3N0ZXJpb3IgPSBsaXN0KG0xX3NlcV9wb3N0ZXJpb3Jfc2FtcGxlcykpCiAgZXZpZGVuY2UgPC0gYmluZF9yb3dzKGV2aWRlbmNlLCBldmlkZW5jZV9zZXEpCn0KCmBgYAoKVGhlIHBsb3QgYmVsb3cgc2hvd3MgdGhlIHNlcXVlbnRpYWwgQmF5ZXMgZmFjdG9ycyByZXN1bHRpbmcgZnJvbSB0aGlzIGFuYWx5c2lzLgooQmF5ZXMgZmFjdG9ycyBjYWxjdWxhdGVkIGZyb20gdmVyeSBzbWFsbCBzYW1wbGVzIHNpemVzIHNob3VsZCBvYnZpb3VzbHkgYmUgdHJlYXRlZCB3aXRoIGNhdXRpb24uKQpUaGUgYWNjdW11bGF0aW9uIG9mIGV2aWRlbmNlIGluIGZhdm91ciBvZiBhbiBlZmZlY3Qgb2YgY29uZGl0aW9uIGlzIGNsZWFyLgpBcyB0aGUgQkYgYmVjb21lcyBsYXJnZXIsIHRoZXJlIGFyZSBzb21lIGxhcmdlIHVwd2FyZCBzcGlrZXM7IHRoaXMgaXMgYmVjYXVzZSBieSB0aGlzIHBvaW50IHRoZXJlIGlzIGVzc2VudGlhbGx5IG5vIHBvc3RlcmlvciBtYXNzIGxlZnQgYXQgemVybywgd2hpY2ggbWFrZXMgdGhlIFNEIGRlbnNpdHkgcmF0aW8gdmVyeSBzZW5zaXRpdmUgdG8gc21hbGwgZmx1Y3R1YXRpb25zIGluIHRoZSBlc3RpbWF0ZWQgcG9zdGVyaW9yIGR1ZSB0byB0aGUgcmFuZG9tbmVzcyBvZiBNQ01DIHNhbXBsaW5nLgoKVGhlIGRheXMgb2YgZGF0YSBjb2xsZWN0aW9uIGFyZSBtYXJrZWQgaW4gdGhlIHBsb3QuIFRoZSBzdG9wcGluZyBydWxlLCB3aGljaCB3YXMgZXZhbHVhdGVkIGF0IHRoZSBlbmQgb2YgZWFjaCBkYXksIHJlcXVpcmVkIHRoZXJlIHRvIGJlIGRhdGEgZnJvbSBhdCBsZWFzdCA0MCBwYXJ0aWNpcGFudHMgKmFuZCogYSBCRiBvZiAxMCBvciBncmVhdGVyIGluIGVpdGhlciBkaXJlY3Rpb24gKHRoaXMgYm91bmRhcnkgaXMgaW5kaWNhdGVkIGJ5IGhvcml6b250YWwgbGluZXMpLgpBdCB0aGUgZW5kIG9mIGRheSAyIHdlIGhhZCBjb2xsZWN0ZWQgZGF0YSBmcm9tIG1vcmUgdGhhbiA0MCBwYXJ0aWNpcGFudHMsIGJ1dCB0aGUgQkYgd2FzIHN0aWxsIHdpdGhpbiB0aGUgYm91bmRhcmllcywgc28gd2UgY29sbGVjdGVkIGRhdGEgZm9yIGFub3RoZXIgZGF5LgoKYGBge3J9CnNlcV9wbG90IDwtIGdncGxvdChmaWx0ZXIoZXZpZGVuY2UsIG4gPiAwKSwgYWVzKG4sIGJmKSkgKwogIGFubm90YXRlKCJyZWN0IiwgeG1pbiA9IDAsIHhtYXggPSAyNywgeW1pbiA9IDEvMTAwLCB5bWF4ID0gMTAwMDAwMCwgZmlsbCA9ICIjMWI5ZTc3IiwgYWxwaGEgPSAwLjIpICsKICBhbm5vdGF0ZSgicmVjdCIsIHhtaW4gPSAyNywgeG1heCA9IDQyLCB5bWluID0gMS8xMDAsIHltYXggPSAxMDAwMDAwLCBmaWxsID0gIiNkOTVmMDIiLCBhbHBoYSA9IDAuMikgKwogIGFubm90YXRlKCJyZWN0IiwgeG1pbiA9IDQyLCB4bWF4ID0gNjgsIHltaW4gPSAxLzEwMCwgeW1heCA9IDEwMDAwMDAsIGZpbGwgPSAiIzc1NzBiMyIsIGFscGhhID0gMC4yKSArCiAgYW5ub3RhdGUoInRleHQiLCBsYWJlbCA9IGMoIkRheSAxIiwgIkRheSAyIiwgIkRheSAzIiksIHggPSBjKDEzLjUsIDM0LjUsIDU1KSwgeSA9IDAuMDIpICsKICBnZW9tX2hsaW5lKHlpbnRlcmNlcHQgPSBjKDAuMSwgMSwgMTApLCBsdHkgPSBjKDEsIDMsIDEpKSArCiAgZ2VvbV92bGluZSh4aW50ZXJjZXB0ID0gYygwLCAyNywgNDIsIDY4KSwgbHR5ID0gMikgKwogIGdlb21fbGluZSgpICsKICBnZW9tX3BvaW50KCkgKwogIHNjYWxlX3lfbG9nMTAobGltaXRzID0gYygxLzEwMCwgMTAwMDAwMCksIGJyZWFrcyA9IGMoMS8xMDAsIDEvMTAsIDEsIDEwLCAxMDAsIDEwMDAsIDEwMDAwLCAxMDAwMDAsIDEwMDAwMDApLCBsYWJlbHMgPSBmdW5jdGlvbihuKSB7Zm9ybWF0KG4sIHNjaWVudGlmaWMgPSBGQUxTRSwgZHJvcDB0cmFpbGluZyA9IFRSVUUpfSkgKwogIGxhYnMoeCA9ICJTYW1wbGUgc2l6ZSIsIHkgPSAiQkYxMCIsIHRpdGxlID0gIlNlcXVlbnRpYWwgQmF5ZXMgZmFjdG9ycyBpbiBmYXZvdXIgb2YgYW4gZWZmZWN0IG9mIGNvbmRpdGlvbiIpCgpzZXFfcGxvdApgYGAKClRoZSBwbG90IGJlbG93IHNob3dzIGhvdyB0aGUgcG9zdGVyaW9yIGZvciB0aGUgY29uZGl0aW9uIGNvZWZmaWNpZW50IGRldmVsb3BzIGFzIHRoZSBzYW1wbGUgc2l6ZSBpbmNyZWFzZXMuClRoZSBkYXNoZWQgdmVydGljYWwgbGluZSByZXByZXNlbnRzIHRoZSBwb2ludCBhdCB3aGljaCB0aGUgcG9zdGVyaW9yIGRlbnNpdHkgaXMgY29tcGFyZWQgdG8gdGhlIHByaW9yIGRlbnNpdHkgKHNob3duIGF0IHRoZSBib3R0b20gaW4gZ3JleSkuCgpgYGB7ciwgZmlnLmhlaWdodD04LCBmaWcud2lkdGg9NX0KdW5uZXN0X2xlZ2FjeShldmlkZW5jZSkgJT4lCiAgZ2dwbG90KGFlcyh4ID0gcG9zdGVyaW9yLCB5ID0gbiwgZ3JvdXAgPSBuLCBmaWxsID0gbiA9PSAwKSkgKwogIGdlb21fdmxpbmUoeGludGVyY2VwdCA9IDAsIGx0eSA9IDIpICsKICBnZW9tX2RlbnNpdHlfcmlkZ2VzMihzY2FsZSA9IDEwLCBzaXplID0gMC4yNSkgKwogIHNjYWxlX3hfY29udGludW91cyhsaW1pdHMgPSBjKC0yLCAyKSkgKwogIGxhYnMoeCA9ICJFc3RpbWF0ZSBmb3IgY29lZmZpY2llbnQgJ2NvbmRpdGlvbiciLCB5ID0gIlNhbXBsZSBzaXplIiwgdGl0bGUgPSAiU2VxdWVudGlhbCBwb3N0ZXJpb3JzIGZvciAnY29uZGl0aW9uJyIpICsKICBzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjKCIjYjNjZGUwIiwgIiM2MzYzNjMiKSkgKwogIGd1aWRlcyhmaWxsID0gRkFMU0UpCmBgYAoKCgojIFNlc3Npb24gaW5mbwpgYGB7cn0Kc2Vzc2lvbkluZm8oKQpgYGAKCg==